EPT: More efficient ept_sync_domain().

Rather than always flushing all CPUs, only flush CPUs this domain is
currently active on, and defer flushing other CPUs until this domain
is scheduled onto them (or the domain is destroyed).

Signed-off-by: George Dunlap <george.dunlap@eu.citrix.com>
Signed-off-by: Keir Fraser <keir.fraser@citrix.com>

diff --git a/xen/arch/x86/domain.c b/xen/arch/x86/domain.c
index 9bfac88412ea048299f1b14b10b08ec9a45c03eb..85e0ba0d59de4a5df0d31978c3f0c27aad8febcb 100644 (file)
--- a/xen/arch/x86/domain.c
+++ b/xen/arch/x86/domain.c
@@ -1328,6 +1328,15 @@ static void __context_switch(void)
         p->arch.ctxt_switch_from(p);
     }
 
+    /*
+     * Mark this CPU in next domain's dirty cpumasks before calling
+     * ctxt_switch_to(). This avoids a race on things like EPT flushing,
+     * which is synchronised on that function.
+     */
+    if ( p->domain != n->domain )
+        cpu_set(cpu, n->domain->domain_dirty_cpumask);
+    cpu_set(cpu, n->vcpu_dirty_cpumask);
+
     if ( !is_idle_vcpu(n) )
     {
         memcpy(stack_regs,
@@ -1336,10 +1345,6 @@ static void __context_switch(void)
         n->arch.ctxt_switch_to(n);
     }
 
-    if ( p->domain != n->domain )
-        cpu_set(cpu, n->domain->domain_dirty_cpumask);
-    cpu_set(cpu, n->vcpu_dirty_cpumask);
-
     gdt = !is_pv_32on64_vcpu(n) ? per_cpu(gdt_table, cpu) :
                                   per_cpu(compat_gdt_table, cpu);
     if ( need_full_gdt(n) )

diff --git a/xen/arch/x86/hvm/vmx/vmx.c b/xen/arch/x86/hvm/vmx/vmx.c
index d19163faec4c1bfd8eb15dfa591089b3fbbdb55c..3e4620fa0eaebcf42b0691152f49cf9da015fb22 100644 (file)
--- a/xen/arch/x86/hvm/vmx/vmx.c
+++ b/xen/arch/x86/hvm/vmx/vmx.c
@@ -72,6 +72,7 @@ static void vmx_fpu_dirty_intercept(void);
 static int vmx_msr_read_intercept(struct cpu_user_regs *regs);
 static int vmx_msr_write_intercept(struct cpu_user_regs *regs);
 static void vmx_invlpg_intercept(unsigned long vaddr);
+static void __ept_sync_domain(void *info);
 
 static int vmx_domain_initialise(struct domain *d)
 {
@@ -91,7 +92,8 @@ static int vmx_domain_initialise(struct domain *d)
 
 static void vmx_domain_destroy(struct domain *d)
 {
-    ept_sync_domain(d);
+    if ( d->arch.hvm_domain.hap_enabled )
+        on_each_cpu(__ept_sync_domain, d, 1);
     vmx_free_vlapic_mapping(d);
 }
 
@@ -666,10 +668,21 @@ static void vmx_ctxt_switch_from(struct vcpu *v)
 
 static void vmx_ctxt_switch_to(struct vcpu *v)
 {
+    struct domain *d = v->domain;
+
     /* HOST_CR4 in VMCS is always mmu_cr4_features. Sync CR4 now. */
     if ( unlikely(read_cr4() != mmu_cr4_features) )
         write_cr4(mmu_cr4_features);
 
+    if ( d->arch.hvm_domain.hap_enabled )
+    {
+        unsigned int cpu = smp_processor_id();
+        /* Test-and-test-and-set this CPU in the EPT-is-synced mask. */
+        if ( !cpu_isset(cpu, d->arch.hvm_domain.vmx.ept_synced) &&
+             !cpu_test_and_set(cpu, d->arch.hvm_domain.vmx.ept_synced) )
+            __invept(1, d->arch.hvm_domain.vmx.ept_control.eptp, 0);
+    }
+
     vmx_restore_guest_msrs(v);
     vmx_restore_dr(v);
     vpmu_load(v);
@@ -1216,11 +1229,20 @@ static void __ept_sync_domain(void *info)
 void ept_sync_domain(struct domain *d)
 {
     /* Only if using EPT and this domain has some VCPUs to dirty. */
-    if ( d->arch.hvm_domain.hap_enabled && d->vcpu && d->vcpu[0] )
-    {
-        ASSERT(local_irq_is_enabled());
-        on_each_cpu(__ept_sync_domain, d, 1);
-    }
+    if ( !d->arch.hvm_domain.hap_enabled || !d->vcpu || !d->vcpu[0] )
+        return;
+
+    ASSERT(local_irq_is_enabled());
+
+    /*
+     * Flush active cpus synchronously. Flush others the next time this domain
+     * is scheduled onto them. We accept the race of other CPUs adding to
+     * the ept_synced mask before on_selected_cpus() reads it, resulting in
+     * unnecessary extra flushes, to avoid allocating a cpumask_t on the stack.
+     */
+    d->arch.hvm_domain.vmx.ept_synced = d->domain_dirty_cpumask;
+    on_selected_cpus(&d->arch.hvm_domain.vmx.ept_synced,
+                     __ept_sync_domain, d, 1);
 }
 
 static void __vmx_inject_exception(int trap, int type, int error_code)

diff --git a/xen/include/asm-x86/hvm/vmx/vmcs.h b/xen/include/asm-x86/hvm/vmx/vmcs.h
index 6695e03c04e2f8c1e99d4dbcadf5e68e877d9b25..f7c3c78cf71ed2b41fddbb9f47091a184d6e48bc 100644 (file)
--- a/xen/include/asm-x86/hvm/vmx/vmcs.h
+++ b/xen/include/asm-x86/hvm/vmx/vmcs.h
@@ -67,6 +67,7 @@ struct vmx_domain {
         };
         u64 eptp;
     } ept_control;
+    cpumask_t ept_synced;
 };
 
 struct arch_vmx_struct {